#!/usr/bin/env python

from google.appengine.ext import db
import bz2, zlib, hashlib, urlparse, logging, base64, struct

def checksum_from_key_name(key_name):
  logging.debug("Translating %s into a checksum." % key_name)
  return base64.urlsafe_b64decode(str(key_name) + "==")

def key_name_from_checksum(checksum):
  return base64.urlsafe_b64encode(checksum).rstrip("=")

def checksum_from_url(url):
  checksum = zlib.crc32(unicode(url).encode("utf-8")) & 0xffffffff
  return struct.pack('>I', checksum)

def key_name_from_url(url):
  checksum = checksum_from_url(url)
  return key_name_from_checksum(checksum)


class Document(db.Model):
  """A URL and its latest PageRank.
  A hash of the URL (the |docid|) is stored in the |key_name|, which allows for
  reverse-lookup of URL hashes.
  """
  
  url = db.TextProperty(required=True) # Not a LinkProperty; can be > 500 bytes
  rank = db.FloatProperty()


class Fetch(db.Expando):
  """This is a repository of fetched documents.
  This inherits from |db.Expando| because headers are dynamically added as 
  properties with a "header-" prefix.
  """
  
  docid = db.ByteStringProperty(required=True)
  date = db.DateTimeProperty(auto_now_add=True)
  status = db.IntegerProperty() # The HTTP status code
  encode = db.StringProperty(
    default="plain",
    choices=["plain", "zlib", "bz2"],
  )
  page = db.BlobProperty()
  
  def __getattr__(self, name):
    if name == 'document':
      return Document.get_by_key_name(key_name_from_checksum(self.docid))
    elif name == 'content': # Perform transparant decompression
      if self.encode == "plain":
        return self.page
      elif self.encode == "bz2":
        return bz2.decompress(self.page)
      elif self.encode == "zlib":
        return zlib.decompress(self.page)
    else:
      return super(Fetch, self).__getattr__(name)
  
  def __setattr__(self, name, value):
    if name == 'document':
      self.docid = checksum_from_key_name(value.key().name())
    elif name == 'content': # Perform transparent compression
      if self.encode == "plain":
        self.page = value
      elif self.encode == "bz2":
        self.page = bz2.compress(value)
      elif self.encode == "zlib":
        self.page = zlib.compress(value)
    else:
      super(Fetch, self).__setattr__(name, value)
  


class RankComputation(db.Model):
  """Metadata for PageRank computations.
  This keeps track of things like how many iterations have been completed, what
  the dampening factor is, when computation began, and how many nodes are in
  the link graph.
  """
  date = db.DateTimeProperty(auto_now_add=True)
  dampening_factor = db.FloatProperty(default=0.85)
  node_count = db.IntegerProperty(default=0)
  iteration_count = db.IntegerProperty(default=0)
  leaks = db.ListProperty(item_type=float)
  residuals = db.ListProperty(item_type=float)


class LinkGraphNode(db.Model):
  """A node in a |RankComputation|'s link graph, and its out-edges."""
  computationid = db.IntegerProperty(required=True)
  docid = db.ByteStringProperty(required=True)
  links = db.BlobProperty(required=True)
  
  def __getattr__(self, name):
    if name == 'document':
      return Document.get_by_key_name(key_name_from_checksum(self.docid))
    elif name == 'computation':
      return Computation.get_by_id(self.computationid)
    else:
      return super(LinkGraphNode, self).__getattr__(name)
  
  def __setattr__(self, name, value):
    if name == 'document':
      self.docid = checksum_from_key_name(value.key().name())
    elif name == 'computation':
      self.computationid = value.key().id()
    else:
      super(LinkGraphNode, self).__setattr__(name, value)


class DocumentRank(db.Model):
  """The accumulated ranks of a |RankComputation| for one |Document|."""
  computationid = db.IntegerProperty(required=True)
  docid = db.ByteStringProperty(required=True)
  dangling = db.BooleanProperty(default=True)
  ranks = db.ListProperty(item_type=float)
  
  def __getattr__(self, name):
    if name == 'document':
      return Document.get_by_key_name(key_name_from_checksum(self.docid))
    elif name == 'computation':
      return RankComputation.get_by_id(self.computationid)
    else:
      return super(DocumentRank, self).__getattr__(name)
  
  def __setattr__(self, name, value):
    if name == 'document':
      self.docid = checksum_from_key_name(value.key().name())
    elif name == 'computation':
      self.computationid = value.key().id()
    else:
      super(DocumentRank, self).__setattr__(name, value)


